{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "思路：\n",
    "1. 模型训练：超参数调优\n",
    "    a) 初步确定弱学习器数目： 20分\n",
    "    b) 对树的最大深度（可选）和min_children_weight进行调优（可选）：20分\n",
    "    c) 对正则参数进行调优：20分\n",
    "    d) 重新调整弱学习器数目：10分\n",
    "    e) 行列重采样参数调整：10分\n",
    "2. 调用模型进行测试10分\n",
    "3. 生成测试结果文件10分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('/Users/user/Documents/AI/W3H/W3H/code/data/RentListingInquries_FE_train.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "x_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 初步确定弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_param = xgb1.get_xgb_params()\n",
    "xgb_param['num_class'] = 9\n",
    "\n",
    "#直接调用xgboost，而非sklarn的wrapper类\n",
    "xgtrain = xgb.DMatrix(x_train, label = y_train)\n",
    "\n",
    "cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=xgb1.get_params()['n_estimators'], folds=list(kfold.split(x_train, y_train)),\n",
    "         metrics='mlogloss', early_stopping_rounds=10)\n",
    "\n",
    "cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "\n",
    "#最佳参数n_estimators\n",
    "n_estimators = cvresult.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "319\n"
     ]
    }
   ],
   "source": [
    "print n_estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对树的最大深度和min_children_weight进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [3, 5, 7, 9], 'min_child_weight': [1, 3, 5]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_dept_child = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_dept_child"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59632, std: 0.00319, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59605, std: 0.00342, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59607, std: 0.00322, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58825, std: 0.00392, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58814, std: 0.00391, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58786, std: 0.00327, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59453, std: 0.00534, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.59247, std: 0.00514, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.59178, std: 0.00436, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.61774, std: 0.00621, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.60855, std: 0.00462, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.60415, std: 0.00372, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 5},\n",
       " -0.58786256569958273)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=319,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_dept_child, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对树的最大深度和min_children_weight进行调优2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth_2 = [4,5,6]\n",
    "min_child_weight_2 = [4,5,6]\n",
    "param_dept_child_2 = dict(max_depth=max_depth_2, min_child_weight=min_child_weight_2)\n",
    "param_dept_child_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59026, std: 0.00397, params: {'max_depth': 4, 'min_child_weight': 4},\n",
       "  mean: -0.59049, std: 0.00381, params: {'max_depth': 4, 'min_child_weight': 5},\n",
       "  mean: -0.59027, std: 0.00416, params: {'max_depth': 4, 'min_child_weight': 6},\n",
       "  mean: -0.58781, std: 0.00513, params: {'max_depth': 5, 'min_child_weight': 4},\n",
       "  mean: -0.58786, std: 0.00327, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58891, std: 0.00364, params: {'max_depth': 5, 'min_child_weight': 6},\n",
       "  mean: -0.59114, std: 0.00356, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.59026, std: 0.00439, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58942, std: 0.00364, params: {'max_depth': 6, 'min_child_weight': 6}],\n",
       " {'max_depth': 5, 'min_child_weight': 4},\n",
       " -0.5878141064500052)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=319,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_dept_child_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, colsample_bylevel=0.7, colsample_bytree=0.8,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
       "       min_child_weight=4, missing=None, n_estimators=319, nthread=-1,\n",
       "       objective='multi:softprob', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=3, silent=True, subsample=0.3)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对正则参数进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 1.5, 2]\n",
    "reg_lambda = [0.5, 1, 2]\n",
    "\n",
    "param_alpha_lamdba = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_alpha_lamdba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58975, std: 0.00404, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58919, std: 0.00387, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58914, std: 0.00378, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58904, std: 0.00363, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58889, std: 0.00430, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58844, std: 0.00418, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 2},\n",
       " -0.58844006283878869)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,\n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3 = GridSearchCV(xgb3, param_grid = param_alpha_lamdba, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3.fit(x_train , y_train)\n",
    "\n",
    "gsearch3.grid_scores_, gsearch3.best_params_,     gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(base_score=0.5, colsample_bylevel=0.7, colsample_bytree=0.6,\n",
      "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
      "       min_child_weight=4, missing=None, n_estimators=1000, nthread=-1,\n",
      "       objective='multi:softprob', reg_alpha=2, reg_lambda=2,\n",
      "       scale_pos_weight=1, seed=3, silent=True, subsample=0.7)\n"
     ]
    }
   ],
   "source": [
    "print gsearch3.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 重新调整弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [2], 'reg_lambda': [2]}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [2]\n",
    "reg_lambda = [2]\n",
    "\n",
    "param_alpha_lamdba = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_alpha_lamdba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb4 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=2000,\n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha = 2,\n",
    "        reg_lambda = 2,\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_param = xgb4.get_xgb_params()\n",
    "xgb_param['num_class'] = 9\n",
    "\n",
    "#直接调用xgboost，而非sklarn的wrapper类\n",
    "xgtrain = xgb.DMatrix(x_train, label = y_train)\n",
    "\n",
    "cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=xgb4.get_params()['n_estimators'], folds=list(kfold.split(x_train, y_train)),\n",
    "         metrics='mlogloss', early_stopping_rounds=10)\n",
    "\n",
    "cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "\n",
    "#最佳参数n_estimators\n",
    "n_estimators = cvresult.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "343\n"
     ]
    }
   ],
   "source": [
    "print n_estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 行列重采样参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       " 'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_sub_col = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_sub_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/user/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58745, std: 0.00362, params: {'subsample': 0.3, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58519, std: 0.00402, params: {'subsample': 0.4, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58389, std: 0.00368, params: {'subsample': 0.5, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58248, std: 0.00318, params: {'subsample': 0.6, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58148, std: 0.00317, params: {'subsample': 0.7, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58124, std: 0.00311, params: {'subsample': 0.8, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58769, std: 0.00355, params: {'subsample': 0.3, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58470, std: 0.00404, params: {'subsample': 0.4, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58371, std: 0.00342, params: {'subsample': 0.5, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58277, std: 0.00340, params: {'subsample': 0.6, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58176, std: 0.00348, params: {'subsample': 0.7, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58048, std: 0.00350, params: {'subsample': 0.8, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58691, std: 0.00367, params: {'subsample': 0.3, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58486, std: 0.00390, params: {'subsample': 0.4, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58343, std: 0.00326, params: {'subsample': 0.5, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58244, std: 0.00319, params: {'subsample': 0.6, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58148, std: 0.00302, params: {'subsample': 0.7, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58088, std: 0.00320, params: {'subsample': 0.8, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58631, std: 0.00383, params: {'subsample': 0.3, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58524, std: 0.00367, params: {'subsample': 0.4, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58301, std: 0.00371, params: {'subsample': 0.5, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58209, std: 0.00373, params: {'subsample': 0.6, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58150, std: 0.00388, params: {'subsample': 0.7, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58053, std: 0.00366, params: {'subsample': 0.8, 'colsample_bytree': 0.9}],\n",
       " {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       " -0.58048235061193088)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=343,\n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha = 2,\n",
    "        reg_lambda = 2,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_sub_col, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5.fit(x_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调用模型进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('/Users/user/Documents/AI/W3H/W3H/code/data/RentListingInquries_FE_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = gsearch5.predict(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 生成测试结果文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "resultData = pd.DataFrame(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "resultData.to_csv('./result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
